import jieba
import datetime

def compute_corpus_dict(sent1, sent2):
    sents = [sent1, sent2]
    texts = [[word for word in jieba.lcut(sent)] for sent in sents]

    all_list = []
    for text in texts:
        all_list += text
    corpus = set(all_list)

    corpus_dict = dict(zip(corpus, range(len(corpus))))
    return corpus_dict

# 建立句子的向量表示
def vector_rep(text, corpus_dict):
    vec = []
    for key in corpus_dict.keys():
        if key in text:
            vec.append((corpus_dict[key], text.count(key)))
        else:
            vec.append((corpus_dict[key], 0))
    vec = sorted(vec, key= lambda x: x[0])
    return vec

from math import sqrt
def similarity_with_2_sents(vec1, vec2):
    inner_product = 0
    square_length_vec1 = 0
    square_length_vec2 = 0
    for tup1, tup2 in zip(vec1, vec2):
        inner_product += tup1[1]*tup2[1]
        square_length_vec1 += tup1[1]**2
        square_length_vec2 += tup2[1]**2

    return (inner_product/sqrt(square_length_vec1*square_length_vec2))

if __name__ == '__main__':
    start = datetime.datetime.now()
    #设置源文件路径
    path = 'data2.txt'
    f = open(path, 'r', encoding='utf-8')
    data_list = f.readlines()
    print('共有 %s 条数据' % len(data_list))
    for i in range(len(data_list)):
        str1 = data_list[i].rstrip('\n')
        if str1 != '000':
            for j in range(len(data_list)):
                if i < j:
                    print('开始第 %s 轮数据处理，%s' % (i, j))
                    str2 = data_list[j].rstrip('\n')
                    if str2 != '000':
                        corpus_dict = compute_corpus_dict(str1, str2)
                        vec1 = vector_rep(str1, corpus_dict)
                        vec2 = vector_rep(str2, corpus_dict)
                        s1 = similarity_with_2_sents(vec1, vec2)
                        if s1 > 0.7:
                            data_list[j] = '000'
    data_list = list(set(data_list))
    data_list.remove('000')
    f.close()
    #设置结果文件名
    result_file = open('result2.txt', 'w')
    print('有效数据共有%s条' % len(data_list))
    for n in data_list:
        result_file.write(n + '\n')
    result_file.close()
    end = datetime.datetime.now()
    print('数据写入成功,开始时间：%' % start)
    print('数据写入成功,结束时间：%' % end)